# -*- coding: utf-8 -*-
import scrapy
from news_project.items import CeItem

# 经济网-食品
class CeFoodSpider(scrapy.Spider):
    name = 'ce_food'
    allowed_domains = ['ce.cn']
    start_urls = ['http://www.ce.cn/cysc/sp/info/']

    def parse(self, response):
        lis = response.xpath("//div[@class='left']//ul/li[not(@class='line')]")
        for li in lis:
            item = CeItem()
            item['title'] = li.xpath("a/text()").get()
            if item['title'] is None:
                continue
            item['push_time'] = li.xpath("text()").getall()[1][3:-1]

            href = li.xpath("a/@href").extract_first()
            yield scrapy.Request(response.urljoin(href), callback=self.parse_content, meta={"data": item})

    def parse_content(self, response):
        self.logger.info('CeFoodSpider-->A response from %s just arrived!', response.url)
        data = response.meta['data']

        content_form = response.xpath("//form[@id='formarticle']")
        source = content_form.xpath("//span[@id='articleSource']/text()").get()
        author = content_form.xpath("//span[@id='articleAuthor']/text()").get()
        content = content_form.xpath("div[@id='articleText']").get()


        data['source'] = source
        data['source_link'] = response.url
        data['author'] = author
        data['content'] = content
        data['module'] = 11

        if data['content'] is None:
            self.logger.warn("Not found the article content! url [%s]", response.url)
        else:
            yield data
